#include <unistd.h>
#include <stdio.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <fts.h>
#include <sched.h>		// sched_setaffinity
#include <signal.h>
#include <fcntl.h>
#include <unistd.h>
#include <errno.h>
#include <string.h>
#include <stdlib.h>
#include <dirent.h>
#include <limits.h>
#include <cpuset.h>
#include <bitops.h>
#include <syslog.h>
#include <sys/socket.h>
#include <linux/netlink.h>
#include <linux/connector.h>
#define _LINUX_TIME_H
#include <linux/cn_proc.h>
#include "public_relate.h"
#include "sched_relate.h"

#define MAX_MSGSIZE 1024

#define LIBINTERFACE

#define SMALL_BUFSZ 8

#define DEFCPUBITS 	1024
#define DEFNODEBITS 1024
static int initial = 0;
static int cpumask_sz;
static int nodemask_sz;
static unsigned long *cpus_allowed_p;
static unsigned long *node_allowed_p;
static char online_cpus_ptr[256];
static char buf_cpulist[256];
static char buf_nodelist[256];

struct cpuset;
struct cpuset_pidlist;
/*
   check_root() checks /cpuset exist or not
RET:
	errno:
*/
static enum {
    check_notdone,
    check_enosys,
    check_enodev,
    check_ok,
} check_state = check_notdone;

static int check(void)
{
    int old_errno = errno;
    if (check_state == check_notdone) {
	struct stat statbuf;
	if (stat("/proc/self/cpuset", &statbuf) < 0) {
	    check_state = check_enosys;
	    goto done;
	}
	if (stat(CHECKTASKS, &statbuf) < 0) {
	    check_state = check_enodev;
	    goto done;
	}
	check_state = check_ok;
    }
  done:
    errno = old_errno;
    switch (check_state) {
    case check_enosys:
	errno = ENOSYS;
	return -1;
    case check_enodev:
	errno = ENODEV;
	return -1;
    default:
	break;
    }
    return 0;
}

/*
 * True if two pathnames resolve to same file.
 * False if either path can not be stat'd,
 * or if the two paths resolve to a different file.
 */

static int samefile(const char *path1, const char *path2)
{
    struct stat sb1, sb2;

    if (stat(path1, &sb1) < 0)	// path1 not exist
    {
#ifdef DEBUG
	printf("%s:%s:%d errno:%d\n", path1, __FILE__, __LINE__, errno);
#endif
	return 0;
    }
    if (stat(path2, &sb2) < 0)	//path1 not exist
    {
#ifdef DEBUG
	printf("%s: %s:%d errno:%d\n", path2, __FILE__, __LINE__, errno);
#endif
	return 0;
    }
    return sb1.st_ino == sb2.st_ino && sb1.st_dev == sb2.st_dev;
}


#define slash(c) (*(c) == '/')
#define eocomp(c) (slash(c) || !*(c))
#define dot1(c) (*(c) == '.' && eocomp(c+1))

/* In place path compression.  Remove extra dots and slashes. */
static char *pathcomp(char *p)
{
    char *a = p;
    char *b = p;

    if (!p || !*p)
	return p;
    if (slash(p))
	*b++ = *a++;
    for (;;) {
	if (slash(a))
	    while (slash(++a))
		continue;
	if (!*a) {
	    if (b == p)
		*b++ = '.';
	    *b = '\0';
	    return (p);
	} else if (dot1(a)) {
	    a++;
	} else {
	    if ((b != p) && !slash(b - 1))
		*b++ = '/';
	    while (!eocomp(a))
		*b++ = *a++;
	}
    }
}

#undef slash
#undef eocomp
#undef dot1

/*
 * pathcat2(buf, buflen, name1, name2)
 *
 * Return buf, of length buflen, with name1/name2 stored in it.
 */

static char *pathcat2(char *buf, int buflen, const char *name1,
		      const char *name2)
{
    if (!name2) {
	snprintf(buf, buflen, "%s/", name1);
	return pathcomp(buf);
    }
    (void) snprintf(buf, buflen, "%s/%s", name1, name2);
    return pathcomp(buf);
}


/*
 * fullpath(buf, buflen, name)
 *
 * Put full path of cpuset 'name' in buffer 'buf'.  If name
 * starts with a slash (``/``) character, then this a path
 * relative to ``/dev/cpuset``, otherwise it is relative to
 * the current tasks cpuset.  Return 0 on success, else
 * -1 on error, setting errno.
 */

static int fullpath(char *buf, int buflen, const char *name)
{
    pathcat2(buf, buflen, cpusetmnt, name);
    pathcomp(buf);
    return 0;
}

static void chomp(char *s)
{
    char *t;

    for (t = s + strlen(s) - 1; t >= s; t--) {
	if (*t == '\n' || *t == '\r')
	    *t = '\0';
	else
	    break;
    }
}

static int read_string_file(const char *path, char *buf, int size)
{
    FILE *fp;
    if ((fp = fopen(path, "r")) == NULL)
		return -1;
    fgets(buf, size, fp);
    fclose(fp);
    return 0;
}

/*
 * Determine number of bytes in a seekable open file, without
 * assuming that stat(2) on that file has a useful size.
 * Has side affect of leaving the file rewound to the beginnning.
 */
static int filesize(FILE * fp)
{
    int sz = 0;
    rewind(fp);
    while (fgetc(fp) != EOF)
	sz++;
    rewind(fp);
    return sz;
}

static void get_online_cpus(void)
{
    FILE *fp;
    int fsize;
    char path[PATH_MAX];
    snprintf(path, PATH_MAX, online_cpu_file);
    if ((fp = fopen(path, "r")) == NULL)
	abort();
    fsize = filesize(fp);
    fgets(online_cpus_ptr, fsize, fp);
    return;
}


/*
 * fullpath2(buf, buflen, name1, name2)
 *
 * Like fullpath(), only concatenate two pathname components on end.
 */

static int fullpath2(char *buf, int buflen, const char *name1,
		     const char *name2)
{
    if (fullpath(buf, buflen, name1) < 0)	/* ==>/MNT/name1 */
	return -1;
    if (strlen(buf) >= buflen - 1 - strlen(name2)) {
	errno = E2BIG;
	return -1;
    }
    strcat(buf, "/");
    strcat(buf, name2);		/* ==> /MNT/name1/name2 */
    pathcomp(buf);
    return 0;
}


/* Write string to file at given filepath.  Create or truncate file. */
static int write_string_file(const char *filepath, const char *str)
{
    int fd = -1;

    if ((fd = open(filepath, O_WRONLY, 0644)) < 0)
        return -1;
    if (write(fd, str, strlen(str)) < 0) {
        close(fd);
        return -1;
    }
    close(fd);
    return 0;
}

static int write_mems(struct cpuset *cs)
{
    char buf[PATH_MAX];
    fullpath2(buf, sizeof(buf), cs->name, MEMSFILE);
    if (!cs->memlist[0])
	strcpy(cs->memlist, buf_nodelist);
    if (write_string_file(buf, cs->memlist) < 0) {
	errno = EINVAL;
	return -1;;
    }
    return 0;
}

static int write_cpus(struct cpuset *cs)
{
    char buf[PATH_MAX];
    fullpath2(buf, sizeof(buf), cs->name, CPUSFILE);
    if (write_string_file(buf, cs->cpulist) < 0) {
	errno = ENOCPU;
	return -1;;
    }
    return 0;
}

/* check a process has flag "flag" or not 
return : FLAG_NO		no flag
	 FLAG_OK		has flag
	 FLAG_NOPID		no such pid
*/
static int test_proc_flag(pid_t pid, unsigned long chflag)
{
    char buf[PATH_MAX];
    char *bp;
    FILE *fp;
    unsigned long flag;
    sprintf(buf, "/proc/%d/stat", pid);
    if ((fp = fopen(buf, "r")) == NULL)
	return FLAG_NOPID;
    if (fgets(buf, sizeof(buf), fp) == NULL)
	return FLAG_NOPID;
    fclose(fp);
    bp = strrchr(buf, ')');
    if (bp)
	sscanf(bp + 1, " %*c %*d %*d %*d %*d %*d %lu", &flag);
#ifdef DEBUG
    printf("process %d flag is %lu\n", pid, flag);
#endif
    if (flag & chflag)
	return FLAG_OK;
    return FLAG_NO;
}

static int __cpuset_bind(pid_t pid, const char *path)
{
    flag_t flg;
    int old_errno;
    char buf[SMALL_BUFSZ];
    snprintf(buf, sizeof(buf), "%u", pid);
    if (write_string_file(path, buf) < 0) {
	if (errno == ENOENT)
	    return -1;		// path error
    }
    /* pid error */
    old_errno = errno;		// store it
    if (errno)
		flg = test_proc_flag(pid, PF_THREAD_BOUND);
    else
		return 0;		// ok bind it;

    errno = old_errno;		// restore it, may be modified by test_proc...
    if (flg == FLAG_OK)		// bound by kernel 
    {
		errno = EBOUND;
		return -1;
    }
    if (flg == FLAG_NOPID) {
		errno = ESRCH;
		return -1;
    }
	return 0;
}

/* test a cpuset is EXCLUDE or not
   @path: full path of a CPUSET
RETURN: 0 not
	1 yes
*/
static int is_exclude(char *path)
{
	char buf[8];
	/* should always success */
	read_string_file(path, buf, 8);
	return buf[0] == '1';
}

static int is_exist(const char *path)
{
    struct stat st;
    int i;
    int old_errno = errno;
    i = stat(path, &st);
    errno = old_errno;
    return i<0 ? 0 : 1;
}

/* Move task (pid == 0 for current) to a cpuset */
LIBINTERFACE int cpuset_bind(pid_t pid, const char *relpath)
{
    char buf[PATH_MAX];
    char dotchar[PATH_MAX];;
    if (check() < 0)
	return -1;
    fullpath2(buf, sizeof(buf), relpath, "tasks");
    /* if not exist, bind a EXCLUDE CPUSERT, add a '.' to head */
    if (!is_exist(buf))
    {
	dotchar[0] = '.';
	dotchar[1] = 0;
	strcat(dotchar, relpath);
    	fullpath2(buf, sizeof(buf), dotchar, "tasks");
    }
    return __cpuset_bind(pid, buf);
}

/* rename a cpuset */
LIBINTERFACE int cpuset_rename(const char *oldname, const char *newname)
{
    char oldbuf[PATH_MAX];
    char newbuf[PATH_MAX];
    char tmp[PATH_MAX];
    fullpath(oldbuf, sizeof(oldbuf), oldname);
    if (is_exist(oldbuf))   /* is a cpuset */
        fullpath(newbuf, sizeof(newbuf), newname);
    else{
        snprintf(tmp, PATH_MAX, ".%s", oldname);
        fullpath(oldbuf, sizeof(oldbuf), tmp);
        snprintf(tmp, PATH_MAX, ".%s", newname);
        fullpath(newbuf, sizeof(oldbuf), tmp);
    }
    return rename(oldbuf, newbuf);
}


/*
 * When recursively reading all the tasks files from a subtree,
 * chain together the read results, one pidblock per tasks file,
 * containing the raw unprocessed ascii as read(2) in.  After
 * we gather up this raw data, we then go back to count how
 * many pid's there are in total, allocate an array of pid_t
 * of that size, and transform the raw ascii data into this
 * array of pid_t's.
 */

struct pidblock {
    char *buf;
    int buflen;
    struct pidblock *next;
};

/* Transform one-per-line ascii pids in pb to pid_t entries in pl */
static int pid_transform(struct pidblock *pb, struct cpuset_pidlist *pl,
			 int n)
{
    char *a, *b;

    for (a = pb->buf; a < pb->buf + pb->buflen; a = b) {
	pid_t p = strtoul(a, &b, 10);
	if (a == b)
	    break;
	pl->pids[n++] = p;
    }
    return n;
}

static void free_pidblocks(struct pidblock *pbhead)
{
    struct pidblock *pb, *nextpb;

    for (pb = pbhead; pb; pb = nextpb) {
	nextpb = pb->next;
	free(pb->buf);
	free(pb);
    }
}

/*
 * Chain the raw contents of a file onto the pbhead list.
 *
 * We malloc "+ 1" extra byte for a nul-terminator, so that
 * the strtoul() loop in pid_transform() won't scan past
 * the end of pb->buf[] and accidentally find more pids.
 */
static void add_pidblock(const char *file, struct pidblock **ppbhead)
{
    FILE *fp = NULL;
    struct pidblock *pb = NULL;
    int fsz;

    if ((fp = fopen(file, "r")) == NULL)
	goto err;
    fsz = filesize(fp);
    if (fsz == 0)
	goto err;
    if ((pb = calloc(1, sizeof(*pb))) == NULL)
	goto err;
    pb->buflen = fsz;
    if ((pb->buf = malloc(pb->buflen + 1)) == NULL)
	goto err;
    if (fread(pb->buf, 1, pb->buflen, fp) > 0) {
	pb->buf[pb->buflen] = '\0';
	pb->next = *ppbhead;
	*ppbhead = pb;
    }
    fclose(fp);
    return;
  err:
#ifdef DEBUG
    printf("%s:%d errno:%d\n", __FILE__, __LINE__, errno);
#endif
    if (fp)
	fclose(fp);
    if (pb)
	free(pb);
}

static void read_task_file(const char *relpath, struct pidblock **ppbhead)
{
    char buf[PATH_MAX];
    char dotchar[PATH_MAX];
    fullpath2(buf, sizeof(buf), relpath, "tasks");
    if (!is_exist(buf))
    {
	dotchar[0] = '.';
	dotchar[1] = 0;
	strcat(dotchar, relpath);
    	fullpath2(buf, sizeof(buf), dotchar, "tasks");
    }
    add_pidblock(buf, ppbhead);
}


/* Count how many pids in buf (one per line - just count newlines) */
static int pidcount(const char *buf, int buflen)
{
    int n = 0;
    const char *cp;

    for (cp = buf; cp < buf + buflen; cp++) {
	if (*cp == '\n')
	    n++;
    }
    return n;
}

/* Return number of elements in pidlist */
static int cpuset_pidlist_length(const struct cpuset_pidlist *pl)
{
    if (pl)
	return pl->npids;
    else
	return 0;
}

/* Return i'th element of pidlist */
static pid_t cpuset_get_pidlist(const struct cpuset_pidlist *pl, int i)
{
    if (pl && i >= 0 && i < pl->npids)
	return pl->pids[i];
    else
	return (pid_t) - 1;
}

/* Free pidlist */
static void cpuset_freepidlist(struct cpuset_pidlist *pl)
{
    if (pl && pl->pids)
	free(pl->pids);
    if (pl)
	free(pl);
}

/* numeric comparison routine for qsort */
static int numericsort(const void *m1, const void *m2)
{
    pid_t p1 = *(pid_t *) m1;
    pid_t p2 = *(pid_t *) m2;

    return p1 - p2;
}

/* Return list pids in cpuset 'path' */
struct cpuset_pidlist *cpuset_init_pidlist(const char *relpath, int flag)
{
    struct pidblock *pb = NULL;
    struct cpuset_pidlist *pl = NULL;
    struct pidblock *pbhead = NULL;
    int n;

    if (check() < 0)
	goto err;

    read_task_file(relpath, &pbhead);
    if (errno)
	return NULL;
    if ((pl = calloc(1, sizeof(*pl))) == NULL)
	goto err;
    pl->npids = 0;
    for (pb = pbhead; pb; pb = pb->next)
	pl->npids += pidcount(pb->buf, pb->buflen);
    if ((pl->pids = calloc(pl->npids, sizeof(pid_t))) == NULL)
	goto err;
    n = 0;
    for (pb = pbhead; pb; pb = pb->next)
	n = pid_transform(pb, pl, n);
    free_pidblocks(pbhead);
    qsort(pl->pids, pl->npids, sizeof(pid_t), numericsort);
    return pl;
  err:
    cpuset_freepidlist(pl);
    free_pidblocks(pbhead);
    return NULL;
}

#define NUMBER_MOVE_TASK_ATTEMPTS 10

LIBINTERFACE int cpuset_bind_cpuset_tasks(const char *fromrelpath,
					  const char *torelpath)
{
    char fromfullpath[PATH_MAX];
    char tofullpath[PATH_MAX];
    int i;
    struct cpuset_pidlist *pl = NULL;
    int sav_errno;
    fullpath(fromfullpath, sizeof(fromfullpath), fromrelpath);
    fullpath(tofullpath, sizeof(tofullpath), torelpath);

    if (samefile(fromfullpath, tofullpath))
	return 0;

    for (i = 0; i < NUMBER_MOVE_TASK_ATTEMPTS; i++) {
	int plen, j;

	if ((pl = cpuset_init_pidlist(fromrelpath, 0)) == NULL) {
	    /* missing cpuset is as good as if all moved */
	    if (errno == ENOENT)
		goto no_more_cpuset;

	    /* other problems reading cpuset are bad news */
	    sav_errno = errno;
	    goto failed;
	}
#ifdef DEBUG
	printf("FILE:%s LINE:%d errno:%d", __FILE__, __LINE__, errno);
#endif
	if ((plen = cpuset_pidlist_length(pl)) == 0)
	    goto no_more_pids;

	for (j = 0; j < plen; j++) {
	    pid_t pid;

	    pid = cpuset_get_pidlist(pl, j);
	    if (cpuset_bind(pid, torelpath) < 0) {
		/* missing task is as good as if moved */
            if (initial)/* when build ..hide cpuset */
                continue;
		if (errno == ESRCH) {
		    errno = 0;
		    continue;
		}
		/* other per-task errors are bad news */
		sav_errno = errno;
		goto failed;
	    }
	}

	cpuset_freepidlist(pl);
	pl = NULL;
    }
    errno = 0;
    return 0;

    /* fall into ... */
  failed:
    cpuset_freepidlist(pl);
    errno = sav_errno;
    return -1;

  no_more_pids:
  no_more_cpuset:
    /* Success - all tasks (or entire cpuset ;) gone. */
//      errno = 0;
    cpuset_freepidlist(pl);
    return 0;
}

/* Get cpuset path of pid into buf
	if pid == 0 return current task's cpuset's path 
*/
LIBINTERFACE char *cpuset_getcpusetpath(pid_t pid, char *buf, size_t size)
{
    int fd;			/* dual use: cpuset file for pid and self */
    int rc;			/* dual use: snprintf and read return codes */
    int e;
    if (check() < 0)
	return NULL;

    /* borrow result buf[] to build cpuset file path */
    rc = snprintf(buf, size, "/proc/%d/cpuset", pid);
    if (rc >= size) {
	errno = E2BIG;
	return NULL;
    }

    if ((fd = open(buf, O_RDONLY)) < 0) {
	e = errno;
	if (e == ENOENT)
	    e = ESRCH;
	if ((fd = open("/proc/self/cpuset", O_RDONLY)) < 0)
	    e = ENOSYS;
	else
	    close(fd);
	errno = e;
	return NULL;
    }
    rc = read(fd, buf, size);
    close(fd);
    if (rc < 0)
	return NULL;
    if (rc >= size) {
	errno = E2BIG;
	return NULL;
    }
    buf[rc - 1] = 0;		//buf[rc-1] = '\n' -> '\0'
    chomp(buf);
    if (rc == 2)		//  like "/\n"
	return buf;
    strncpy(buf, buf + 1, rc + 1);	// strip leading '/' like: "/xx --> xx" 
    return buf;
}

/*delete a cpuset which pid runs in it*/
LIBINTERFACE int cpuset_del_bypid(pid_t pid)
{
    char buf[PATH_MAX];
    if (cpuset_getcpusetpath(pid, buf, sizeof(buf)) == NULL)
	return -1;
    return cpuset_del_byname(buf, 1);
}

/* delete a cpuset, if tasks is not empty and 
   force == 1 ,delete it anyway
*/
LIBINTERFACE int cpuset_del_byname(const char *name, int force)
{
    char buf[PATH_MAX];
    if (!strcmp(name, "/"))	// "/" cpuset
    {
	errno = EINVAL;
	return -1;
    }

    if (fullpath(buf, sizeof(buf), name) < 0)
	return -1;
    if (rmdir(buf) < 0) {
#ifdef DEBUG
	printf("LINE:%d \terrno:%d\n ", __LINE__, errno);
#endif
	if (force)
	    goto force;
	else
	    return -1;
    }
    return 0;
  force:
    errno = 0;
    if (cpuset_bind_cpuset_tasks(name, NULL) < 0)	// move to root cpuset
	return -1;
    return rmdir(buf);
}

/* 
   show the cpuset named "name" 's pid.
   stored at pidarray
DESC:
	name: the target cpuset name
	pl: pids store in
RETURN:
	if return NULL, error happened, check errno.
****MUST USE PARES WITH cpuset_showpid_end()****
*/
LIBINTERFACE struct cpuset_pidlist *cpuset_getpid_begin(const char *name)
{
    char buf[PATH_MAX];
    struct cpuset_pidlist *pl;
    strcpy(buf, name);		//copy name
    if ((pl = cpuset_init_pidlist(buf, 0)) == NULL)
	return NULL;
    if (cpuset_pidlist_length(pl) == 0)
	goto no_more_pids;
    return pl;
  no_more_pids:
    cpuset_freepidlist(pl);
    errno = 0;
    return NULL;
}

/* 
   free the memory malloced by
   cpuset_getpid_begin()
DESC:
	pl: point returned by the cpuset_getpid_begin()
	**** MUST USE PARES WITH cpuset_getpid_begin()****
*/
LIBINTERFACE void cpuset_getpid_end(struct cpuset_pidlist *pl)
{
    cpuset_freepidlist(pl);
}

static inline int streq(const void *s1, const void *s2)
{
    return !strcmp(s1, s2);
}

/* Return true if path is a directory. */
static int isdir(const char *path)
{
    struct stat statbuf;

    if (stat(path, &statbuf) < 0)
	return 0;
    return S_ISDIR(statbuf.st_mode);
}

/* read a file to buf
   path is a buf to fill with 
   */
static int readfile(char *buf, size_t size)
{
    FILE *fp = NULL;
    int fsz;

    if ((fp = fopen(buf, "r")) == NULL)
	goto err;
    fsz = filesize(fp);
    if (fsz == 0)
	goto err;
    if (fread(buf, 1, fsz, fp) > 0) {
	buf[fsz] = 0;
    }
    fclose(fp);
    return fsz;
  err:
    if (fp)
	fclose(fp);
    return 0;
}

/*
   show all the cpuset's name
*/
LIBINTERFACE struct cpuset_namelist *cpuset_getname_begin()
{
    DIR *dir = NULL;
    struct dirent *dent;
    char buf[PATH_MAX];
    int num = 0;
    ssize_t size = 0;
    struct cpuset_namelist *nl;
    if (check() < 0)
	goto err;
    if ((dir = opendir(cpusetmnt)) == NULL)
	goto err;
    while ((dent = readdir(dir)) != NULL) {
	//      if (streq(dent->d_name, ".") || streq(dent->d_name, ".."))
	if (dent->d_name[0] == '.')	/* start with '.', contains ".", ".." ".XX" */
	    continue;
	fullpath(buf, sizeof(buf), dent->d_name);
	if (!isdir(buf))
	    continue;
	++num;
    }				// just get the num 
    nl = malloc(sizeof(*nl));
    nl->num = num;
    nl->name = malloc(PATH_MAX * num);
    nl->memlist = malloc(PATH_MAX * num);
    nl->cpulist = malloc(PATH_MAX * num);
    rewinddir(dir);		// from beginning
    while ((dent = readdir(dir)) != NULL) {
	//      if (streq(dent->d_name, ".") || streq(dent->d_name, ".."))
	if (dent->d_name[0] == '.')	/* start with '.', contains ".", ".." ".XX" */
	    continue;
	fullpath(buf, sizeof(buf), dent->d_name);
	if (!isdir(buf))
	    continue;
	strncpy(nl->name[--num], dent->d_name, dent->d_reclen + 1);	// +1 for nul terminater

	// copy memlist
	fullpath2(buf, sizeof(buf), dent->d_name, MEMSFILE);
	size = readfile(buf, sizeof(buf));
	buf[size - 1] = 0;	// buf[size-1] = '\n'
	strncpy(nl->memlist[num], buf, size);

	// copy cpulist
	fullpath2(buf, sizeof(buf), dent->d_name, CPUSFILE);
	size = readfile(buf, sizeof(buf));
	buf[size - 1] = 0;	// buf[size] = '\n'
	strncpy(nl->cpulist[num], buf, size);

    }
    qsort(nl->name, nl->num, PATH_MAX, streq);
#ifdef DEBUG
    for (num = 0; num < nl->num; num++)
	printf("copy:%s\n", nl->name[num]);
#endif
    return nl;
  err:
    return NULL;
}

LIBINTERFACE void cpuset_getname_end(struct cpuset_namelist *nl)
{
    if (nl && !nl->name)
	free(nl->name);
    if (!nl)
	free(nl);
}


/*
 * Convert the string length of an ascii hex mask to the number
 * of bits represented by that mask.
 *
 * The cpumask and nodemask values in /proc/self/status are in an
 * ascii format that uses 9 characters for each 32 bits of mask.
 */
static int s2nbits(const char *s)
{
    return strlen(s) * 32 / 9;
}

/* Is string 'pre' a prefix of string 's'? */
static int strprefix(const char *s, const char *pre)
{
    return strncmp(s, pre, strlen(pre)) == 0;
}

static int get_allowed_p(void)
{
    FILE *fp = NULL;
    if ((fp = fopen(cpusetmnt CPUSFILE, "r")) == NULL)
	goto fail;
    fgets(buf_cpulist, sizeof(buf_cpulist), fp);
    fclose(fp);
    if ((fp = fopen(cpusetmnt MEMSFILE, "r")) == NULL)
	goto fail;
    fgets(buf_nodelist, sizeof(buf_nodelist), fp);
    fclose(fp);
    /* malloc a buf to store the bits */
    cpus_allowed_p = malloc(cpumask_sz / sizeof(unsigned long));
    node_allowed_p = malloc(nodemask_sz / sizeof(unsigned long));	//???
    bitmap_parselist(buf_cpulist, cpus_allowed_p, cpumask_sz);
    bitmap_parselist(buf_nodelist, node_allowed_p, nodemask_sz);
#ifdef DEBUG
    printf("cpu&&mem list:%s%s", buf_cpulist, buf_nodelist);
#endif
    return 0;
  fail:
    return -1;			// errno seted
}

/* get size of cpumask */
static void update_mask_sizes()
{
    FILE *fp = NULL;
    char *buf = NULL;
    int fsize;
    if ((fp = fopen(mask_size_file, "r")) == NULL)
	goto done;
    fsize = filesize(fp);
    if ((buf = malloc(fsize)) == NULL)
	goto done;

    /*
     * Beware: mask sizing arithmetic is fussy.
     * The trailing newline left by fgets() is required.
     */
    while (fgets(buf, fsize, fp)) {
	if (strprefix(buf, cpumask_prefix))
	    cpumask_sz = s2nbits(buf + strlen(cpumask_prefix));
	if (strprefix(buf, nodemask_prefix))
	    nodemask_sz = s2nbits(buf + strlen(nodemask_prefix));
    }

#ifdef DEBUG
    printf("cpus:%d nodes:%d\n", cpumask_sz, nodemask_sz);
#endif
  done:
    if (buf != NULL)
	free(buf);
    if (fp != NULL)
	fclose(fp);
    if (cpumask_sz == 0)
	cpumask_sz = DEFCPUBITS;
    if (nodemask_sz == 0)
	nodemask_sz = DEFNODEBITS;
}

/* Number of bits in a CPU bitmask on current system */
int cpuset_cpus_nbits()
{
    if (cpumask_sz == 0)
	update_mask_sizes();
    return cpumask_sz;
}

/* Number of bits in a Memory bitmask on current system */
int cpuset_mems_nbits()
{
    if (nodemask_sz == 0)
	update_mask_sizes();
    return nodemask_sz;
}


/* Write string to file at given filepath.  Create or truncate file. */
static int file_write_bysize(const char *filepath, const char *str,
			     size_t size)
{
    int fd = -1;
    int ret = 0;
    errno = 0;
    if ((fd = open(filepath, O_WRONLY, 0644)) < 0)
	goto err_nofd;
    ret = write(fd, str, size);
    if ( ret < 0)
	goto err;
    close(fd);
    return 0;
  err:
    if (fd >= 0)
	close(fd);
  err_nofd:
    if (errno == ENOENT);	/* do nothing */
    else
	errno = ENOCPU;		/* no pid has checked before this function */
    return -1;
}

/* show lbarea list */
LIBINTERFACE char *cpuset_lblist(void)
{
    char *buf = NULL;
    int size;
    FILE *fp;
    if (check() < 0)
	return NULL;
    if ((fp = fopen(cpusetmnt lbarea_file, "r")) == NULL)
	return NULL;
    size = filesize(fp);
    if ((buf = malloc(size + 1)) == NULL)
	return NULL;
    fread(buf, 1, size, fp);
    buf[size] = 0;
    return buf;
}

/* set cpulist to a cpuset
   if path error-> errno == ENOENT
   if cpulist error -> errno == EINVAL, change to ENOCPU
 */
LIBINTERFACE int cpuset_setcpulist(const char *name, const char *cpulist)
{
    char path[PATH_MAX];
    char tmp[PATH_MAX];
    char *hide_cpulist_back;
    char *hide_cpulist;
    char *oldlist;
    unsigned long *bitmaskp;
    unsigned long *oldmaskp;
    unsigned long *allmaskp;
    unsigned long *valid_cpus_maskp;
    /* set a normal cpuset with cpulist */
    fullpath2(tmp, sizeof(tmp), name, CPUSFILE);
    if (is_exist(tmp))
    {
        if (write_string_file(tmp, cpulist) < 0) {
            if (errno == EINVAL)
                errno = ENOCPU;
            goto err;
        }
		/* name exist, set it and return */
		return 0;
    }
    /*not a normal cpuset, may be a exclusive cpuset or NOENT */
    sprintf(path, ".%s", name);
    fullpath2(tmp, sizeof(tmp), path, CPUSFILE);
    if (!is_exist(tmp))
    {
        errno = ENOENT;
        goto err;
    }
    /*OK, it's a exclusive cpuset */
    get_online_cpus();
    update_mask_sizes();
    hide_cpulist = malloc(cpumask_sz);
    oldlist= malloc(cpumask_sz);
    oldmaskp= malloc(cpumask_sz / sizeof(unsigned long));
    allmaskp= malloc(cpumask_sz / sizeof(unsigned long));
    bitmaskp = malloc(cpumask_sz / sizeof(unsigned long));
    valid_cpus_maskp = malloc(cpumask_sz / sizeof(unsigned long));
    /* compare cs->cpulist is legal or not */
    bitmap_parselist(cpulist, bitmaskp, cpumask_sz);
    bitmap_parselist(online_cpus_ptr, valid_cpus_maskp, cpumask_sz);
    if (!bitmap_subset(bitmaskp, valid_cpus_maskp, cpumask_sz)) {
        errno = ENOCPU;
        goto err;
    }
    /* read the old NAME's cpulist */
    read_string_file(tmp, oldlist, PATH_MAX);
    bitmap_parselist(oldlist, oldmaskp, cpumask_sz);
    /* get ..hide/cpuset.cpus and make a backup */
    fullpath2(path, PATH_MAX, HIDE, CPUSFILE);
    read_string_file(path, hide_cpulist, PATH_MAX);
    hide_cpulist_back = strdup(hide_cpulist);
    bitmap_parselist(hide_cpulist, valid_cpus_maskp, cpumask_sz);
    /*final hide = hide | old */
    bitmap_or(allmaskp, valid_cpus_maskp, oldmaskp, cpumask_sz);
    /*then check subset or not */
    if (!bitmap_subset(bitmaskp, allmaskp, cpumask_sz)) {
        errno = ENOCPU;
        goto err;
    }
    if (!bitmap_intersects(oldmaskp, bitmaskp, cpumask_sz))
    {
        /*1st: get hide cpumask: hidemask
2nd: make an AND mathematic: hidemask &= bitmaskp
3th: write hidemask to HIDEFILE
4th: maks a OR mathematic :hidemask |= oldmaskp
5th: write hidemask to HIDEFILE
        get new mask for ..hide */
        bitmap_andnot(valid_cpus_maskp, valid_cpus_maskp, bitmaskp,
              cpumask_sz);
        /* convert bitmask to string 1st */
        bitmap_scnlistprintf(hide_cpulist, cpumask_sz, valid_cpus_maskp,
                 cpumask_sz);
        /* write to ..hide/cpuset.cpus */
        write_string_file(path, hide_cpulist);
        /* write the cpulist to Ecpuset */
        if (write_string_file(tmp, cpulist) < 0)
        {
            write_string_file(path, hide_cpulist_back);
            errno = EINVAL;
            goto err;
        }
        /* convert bitmask to string 2st */
        bitmap_or(valid_cpus_maskp, valid_cpus_maskp, oldmaskp,
                cpumask_sz);
        bitmap_scnlistprintf(hide_cpulist, cpumask_sz, valid_cpus_maskp,
                 cpumask_sz);
        write_string_file(path, hide_cpulist);
        return 0;
    }
    /* bitmaskp and oldmaskp  intersects, a bit complextion...
H:hidemask
O:original mask of name
N:new mask of name
T: temp mask
    precondition:   O & N != NULL
1st: T= N & O
2nd: T= N & ~T
3th: H= H & ~T
3th: W(H)
4th: W(N)
5th: T= N & O
6th: T=O & ~T
7th: H= H | T
8th: W(H)
       */
    bitmap_and(allmaskp, bitmaskp, oldmaskp, cpumask_sz);
    bitmap_andnot(allmaskp, bitmaskp, allmaskp, cpumask_sz);
    bitmap_andnot(valid_cpus_maskp, valid_cpus_maskp, allmaskp,
                 cpumask_sz);
    bitmap_scnlistprintf(hide_cpulist, cpumask_sz, valid_cpus_maskp,
                 cpumask_sz);
    write_string_file(path, hide_cpulist);
    if (write_string_file(tmp, cpulist) < 0)
    {
        write_string_file(path, hide_cpulist_back);
        errno = EINVAL;
        goto err;
    }
    bitmap_and(allmaskp, bitmaskp, oldmaskp, cpumask_sz);
    bitmap_andnot(allmaskp, oldmaskp, allmaskp, cpumask_sz);
    bitmap_or(valid_cpus_maskp, valid_cpus_maskp, allmaskp, cpumask_sz);
    bitmap_scnlistprintf(hide_cpulist, cpumask_sz, valid_cpus_maskp,
                 cpumask_sz);
    write_string_file(path, hide_cpulist);
    return 0;
  err:
    free(hide_cpulist);
    free(bitmaskp);
    free(valid_cpus_maskp);
    return -1;
}

static int cpuset_mark_lbarea(const char *name)
{
    char buf[PATH_MAX];
    int ret;
    int old_errno = errno;
    fullpath2(buf, sizeof(buf), name, EXCLUDEFILE);
    ret = write_string_file(buf, "1");
    errno = old_errno;
    return ret < 0 ? -1 : 0;
}
static int cpuset_unmark_lbarea(const char *name)
{
    char buf[PATH_MAX];
    fullpath2(buf, sizeof(buf), name, EXCLUDEFILE);
    write_string_file(buf, "0");
    errno = 0;
    return 0;
}

/* Rewind cpuset list to beginning */
static void cpuset_fts_rewind(struct cpuset_fts_tree *cs_tree)
{
    if (cs_tree)
	cs_tree->next = cs_tree->head;
}

struct cpuset_fts_tree *cpuset_fts_open(const char *cpusetpath)
{
    FTS *fts = NULL;
    FTSENT *ftsent;
    char *path_argv[2];
    char buf[PATH_MAX];
    struct cpuset_fts_tree *cs_tree = NULL;
    struct cpuset_fts_entry *ep = NULL;
    struct cpuset_fts_entry **pnlep;
    int fts_flags;
    int size;
    fullpath(buf, sizeof(buf), cpusetpath);
    path_argv[0] = buf;
    path_argv[1] = NULL;

    fts_flags = FTS_PHYSICAL | FTS_NOCHDIR | FTS_NOSTAT | FTS_XDEV;
    fts = fts_open(path_argv, fts_flags, NULL);
    if (fts == NULL)
	goto err;

    cs_tree = malloc(sizeof(*cs_tree));
    if (cs_tree == NULL)
	goto err;
    pnlep = &cs_tree->head;
    *pnlep = NULL;

    while ((ftsent = fts_read(fts)) != NULL) {
	if (ftsent->fts_info != FTS_D && ftsent->fts_info != FTS_DNR)
	    continue;
	/* jump root cpuset */
	if (!ftsent->fts_level)
	    continue;
	/* jump ..hide and normal cpuset */
	if (!strcmp("..hide", ftsent->fts_name) ||
	    ftsent->fts_name[0] != '.')
	    continue;
	ep = malloc(sizeof(*ep));
	if (ep == NULL)
	    goto err;
	*pnlep = ep;
	pnlep = &ep->next;
	ep->cpuset = malloc(sizeof(struct cpuset));
	if (ep->cpuset == NULL)
	    goto err;
	// copy cpulist
	fullpath2(buf, sizeof(buf), ftsent->fts_name, CPUSFILE);
	size = readfile(buf, sizeof(buf));
	buf[size - 1] = 0;	// buf[size] = '\n'
	strncpy(ep->cpuset->cpulist, buf, size);
	strcpy(ep->cpuset->name, ftsent->fts_name);

#ifdef DEBUG
	printf("%d:name:%s path:%s\n", ftsent->fts_level, ftsent->fts_name,
	       ftsent->fts_path);
#endif

    }
    if (ep)
	ep->next = NULL;	/* or *pnlep = NULL */
    fts_close(fts);
    cpuset_fts_rewind(cs_tree);
    return cs_tree;
  err:
    if (cs_tree);
    if (fts)
	fts_close(fts);
    return NULL;
}

/* Return pointer to next cpuset entry in hierarchy */
const struct cpuset_fts_entry *cpuset_fts_read(struct cpuset_fts_tree
					       *cs_tree)
{
    const struct cpuset_fts_entry *cs_entry;
    if (cs_tree)		/* check NULL */
	cs_entry = cs_tree->next;
    else
	return NULL;
    if (cs_tree->next != NULL)	/* seek to next entry */
	cs_tree->next = cs_tree->next->next;
    return cs_entry;
}

LIBINTERFACE int cpuset_showlbarea()
{
    struct cpuset_fts_tree *cs_tree;
    const struct cpuset_fts_entry *cs_entry;
    cs_tree = cpuset_fts_open(".");
    while ((cs_entry = cpuset_fts_read(cs_tree)) != NULL)
	fprintf(stdout, "%s\t\t%s\n", cs_entry->cpuset->name + 1,	// +1 skip the '.' 
		cs_entry->cpuset->cpulist);
    get_online_cpus();
    update_mask_sizes();
    unsigned long *valid_cpus_maskp = malloc(cpumask_sz / sizeof(unsigned long));
    unsigned long *maskp= malloc(cpumask_sz / sizeof(unsigned long));
    char *hide_cpulist = malloc(cpumask_sz);
    char path[PATH_MAX];
    bitmap_parselist(online_cpus_ptr, valid_cpus_maskp, cpumask_sz);
    /* get ..hide/cpuset.cpus and make a backup */
    fullpath2(path, PATH_MAX, HIDE, CPUSFILE);
    if (!is_exist(path))
    {
        *hide_cpulist = 0;
        goto ret;
    }
    read_string_file(path, hide_cpulist, PATH_MAX);
    bitmap_parselist(hide_cpulist, maskp, cpumask_sz);
    /* get exclusived mask */
    bitmap_andnot(maskp, valid_cpus_maskp, maskp, cpumask_sz);
    /* convert bitmask to string */
    bitmap_scnlistprintf(hide_cpulist, cpumask_sz, maskp, cpumask_sz);
ret:
    fprintf(stdout, "exclusived cpu:%s\n", hide_cpulist);
    fprintf(stdout, "online cpu:    %s\n", online_cpus_ptr);
    free(hide_cpulist);
    free(valid_cpus_maskp);
    free(maskp);
	return 0;
}

/* del a load balance area */
LIBINTERFACE int cpuset_dellbarea(const char *name)
{
    struct cpuset_fts_tree *cs_tree;
    const struct cpuset_fts_entry *cs_entry;
    char *delname = NULL;
    char *delcpulist;
    char path[PATH_MAX];
    char *hide_cpulist;
    unsigned long *bitmaskhide;
    unsigned long *bitmaskdel;
    int ret;
    cs_tree = cpuset_fts_open(".");
    while ((cs_entry = cpuset_fts_read(cs_tree)) != NULL)
	if (!strcmp(name, cs_entry->cpuset->name + 1)) {
	    delname = cs_entry->cpuset->name;
	    delcpulist = cs_entry->cpuset->cpulist;
	    break;
	}
    if (!delname) {
	errno = ENOENT;
	return -1;
    }

    /* get name and list, we optimise to add to ..hide first , then del it */
    update_mask_sizes();
    bitmaskhide = malloc(cpumask_sz);
    bitmaskdel = malloc(cpumask_sz);
    hide_cpulist = malloc(cpumask_sz);

    fullpath2(path, PATH_MAX, HIDE, CPUSFILE);
    read_string_file(path, hide_cpulist, PATH_MAX);

    bitmap_parselist(hide_cpulist, bitmaskhide, cpumask_sz);
    bitmap_parselist(delcpulist, bitmaskdel, cpumask_sz);
    bitmap_or(bitmaskhide, bitmaskhide, bitmaskdel, cpumask_sz);

    bitmap_scnlistprintf(hide_cpulist, cpumask_sz, bitmaskhide,
			 cpumask_sz);
    /* del it */
    ret = cpuset_del_byname(delname, 1);
    if (ret < 0)    return -1;
    /* write back the cpulist */
    write_string_file(path, hide_cpulist);
    return 0;
}

/* del all load balance area */
LIBINTERFACE void cpuset_dellbarea_all(void)
{
    struct cpuset_fts_tree *cs_tree;
    const struct cpuset_fts_entry *cs_entry;
    char path[PATH_MAX];

    cs_tree = cpuset_fts_open(".");
    while ((cs_entry = cpuset_fts_read(cs_tree)) != NULL) {
        if (cpuset_del_byname(cs_entry->cpuset->name, 1) < 0)
            return;
    }
    get_online_cpus();
    fullpath2(path, PATH_MAX, HIDE, CPUSFILE);
    write_string_file(path, online_cpus_ptr);
}

static void sendmsg_to_kernel(struct kernel_relate_msg *krm)
{
	int sd;
	struct sockaddr_nl daddr;
	int group;
	struct nlmsghdr *nlhdr = NULL;
	struct msghdr msg;
	struct iovec iov;
	struct cn_msg *cnmsg;
	int ret;
	size_t size = sizeof(*krm);
	group = CN_IDX_RELATE_TOKERNEL;
	sd = socket(PF_NETLINK, SOCK_RAW, NETLINK_CONNECTOR);
	if (sd < 0)
	{
	    syslog(LOG_USER|LOG_INFO, "%s: socket open error\n", "cpuset");
        goto out;
	}
	nlhdr = (struct nlmsghdr *) malloc (NLMSG_SPACE (MAX_MSGSIZE));
	if (nlhdr == NULL)
		goto out;
	daddr.nl_family = AF_NETLINK;
	daddr.nl_pid = 0;
	daddr.nl_groups = CN_IDX_RELATE_TOKERNEL;
	if (bind(sd, (struct sockaddr *) &daddr, sizeof (daddr)) < 0)
	{
	    syslog(LOG_USER|LOG_INFO, "%s: bind error\n", "cpuset");
		goto ret_free;
	}
	ret = setsockopt(sd, SOL_NETLINK,1, &group, sizeof(group));
	if (ret < 0)
	{
	  	syslog(LOG_USER|LOG_INFO, "%s: setsocket error\n", "cpuset");
		goto ret_free;
	}
	memset (nlhdr, 0, sizeof (NLMSG_SPACE (MAX_MSGSIZE)));
	memset (&iov, 0, sizeof (struct iovec));
	memset (&msg, 0, sizeof (struct msghdr));
	cnmsg = (struct cn_msg *) NLMSG_DATA (nlhdr);
	memcpy(cnmsg->data, krm, size);
	nlhdr->nlmsg_len = NLMSG_LENGTH (sizeof (struct cn_msg) + size);
	nlhdr->nlmsg_pid = 0;
	nlhdr->nlmsg_flags = 0;
	cnmsg->id.idx = CN_IDX_RELATE_TOKERNEL;
	cnmsg->id.val = CN_VAL_RELATE_TOKERNEL;
	cnmsg->seq = 0;
	cnmsg->ack = 0;
	cnmsg->len = size;	/* user_message length */
	iov.iov_base = (void *) nlhdr;
	iov.iov_len = nlhdr->nlmsg_len;
	msg.msg_name = (void *) &daddr;
	msg.msg_namelen = sizeof (daddr);
	msg.msg_iov = &iov;
	msg.msg_iovlen = 1;
	ret = sendmsg(sd, &msg, 0);
	if (ret < 0)
	  syslog(LOG_USER|LOG_INFO,"%s: sendmsg error\n", "cpuset");
ret_free:
	free(nlhdr);
out:
	close(sd);
	return ;
}

/* migrate a process to run in a cpu 
	get process's CPUs allowed mask from /proc/pid/status
	coz: when migrate a process to a cpu, change it's mask
	while the second time to migrate to another cpu, rewrite
	this mask. */

LIBINTERFACE int cpuset_migrate(pid_t pid, int cpu)
{
    struct kernel_relate_msg krm;
    unsigned long *bitmaskp = malloc(cpumask_sz / sizeof(unsigned long));

    update_mask_sizes();
    bitmaskp = malloc(cpumask_sz / sizeof(unsigned long));
   /* test pid */
    flag_t flg;
    flg = test_proc_flag(pid, PF_THREAD_BOUND);
    if (flg == FLAG_OK)		// bound by kernel 
    {
	errno = EBOUND;
	return -1;
    }
    if (flg == FLAG_NOPID) {
	errno = ESRCH;
	return -1;
    }
    /* test cpu */
    if (cpu < 0) {
	errno = ENOCPU;
	return -1;
    }
    /* get the global allowed mask */
    if (get_allowed_p() < 0)
	goto fail;

    if (!test_bit(cpu, cpus_allowed_p)) {
	errno = ENOCPU;
	goto fail;
    }
    
//#	if (cpuset_getcpusetpath(pid, path, sizeof(path)) == NULL)
//#			goto fail;
//#
//#	fullpath2(buf, sizeof(buf), path, CPUSFILE);
//#	if ((fp = fopen(buf, "r")) == NULL)
//#			goto fail;	// errno seted
//#	fread(listbuf, PATH_MAX, 1, fp);
//#	bitmap_parselist(listbuf, bitmaskp, cpumask_sz);
//#	/* cpu in this cpuset, just move this process to cpu,
//#	   if destcpu == cruuent cpu, the kernel will do it for us */
//#	if (!test_bit(cpu, bitmaskp))
//#	{
//#		/* move to "/" and a system call to bind cpus */
//#		cpuset_bind(pid, "/");
//#		// don't use cpu_set(cpu, bitmaskp); migrate will do it
//#		int i;
//#		cpu_set_t mask;
//#		sched_getaffinity(pid, sizeof(cpu_set_t),&mask); 
//#		set_bit(11, &mask);
//#		i = sched_setaffinity(pid, sizeof(cpu_set_t), &mask);
//#		printf("ret:%d %d\n", i, sizeof(cpu_set_t));
//#	}
	krm.magic = RELATE_MAGIC;
	krm.pid = pid;
    krm.latestcpu = cpu;
	krm.what = MIGRATE;
#ifdef RELATE_DEBUG
    krm.d = 1;
#endif
    sendmsg_to_kernel(&krm);
    return 0;
  fail:
    return -1;
}


/* kill all the process in a cpuset 
   if some process can't be killed ,move it to / 
   when a pid < PID_MIN, do not kill it.not implement
   */
LIBINTERFACE int cpuset_kill(const char *name, int force)
{
    struct cpuset_pidlist *pl = NULL;
    int plen;
    int sav_errno = 0;
    int j;
    pid_t pidx;
    flag_t flag;
    if ((pl = cpuset_init_pidlist(name, 0)) == NULL) {
	/* missing cpuset is as good as if all moved */
	if (errno == ENOENT)
	    goto no_more_cpuset;

	/* other problems reading cpuset are bad news */
	sav_errno = errno;
	goto failed;
    }
    if ((plen = cpuset_pidlist_length(pl)) == 0)
	goto no_more_pids;
    for (j = 0; j < plen; j++) {
	pidx = cpuset_get_pidlist(pl, j);
	if (kill(pidx, SIGKILL) < 0) {
	    if (errno == ESRCH)
		errno = 0;	// porcess may exit
	}
	/* send a SIGKILL, but some process exiting slowly
	   and some process are not killable, check their status
	   and move them to "/"
	 */
	flag = test_proc_flag(pidx, PF_EXITING | PF_SIGNALED);
	switch (flag) {
	case FLAG_NO:
	    cpuset_bind(pidx, "/");
	    break;
	case FLAG_NOPID:
	    errno = 0;		/* clean it, made by the test_proc_flag() func */
	    //follow down
	case FLAG_OK:
	    //follow down
	default:
	    break;
	}

#ifdef DEBUG
	printf("LINE:%d kill pid:%d \terrno:%d\n", __LINE__, pidx, errno);
#endif
    }
  no_more_cpuset:
  no_more_pids:
    cpuset_freepidlist(pl);
    if (force)
	cpuset_del_byname(name, 1);

#ifdef DEBUG
    printf("LINE:%d kill pid:%d \terrno:%d\n ", __LINE__, pidx, errno);
#endif
    return 0;

  failed:
    cpuset_freepidlist(pl);
    errno = sav_errno;
    return -1;

}

/*
 * int cpuset_latestcpu(pid_t pid)
 *
 * Return most recent CPU on which task pid executed.  If pid == 0,
 * examine current task.
 *
 * The last used CPU is visible for a given pid as field #39 (starting
 * with #1) in the file /proc/pid/stat.  Currently this file has 41
 * fields, in which case this is the 3rd to the last field.
 *
 * Unfortunately field #2 is a command name and might have embedded
 * whitespace.  So we can't just count white space separated fields.
 * Fortunately, this command name is surrounded by parentheses, as
 * for example "(sh)", and that closing parenthesis is the last ')'
 * character in the line.  No remaining fields can have embedded
 * whitespace or parentheses.  So instead of looking for the 39th
 * white space separated field, we can look for the 37th white space
 * separated field past the last ')' character on the line.
 */

/* Return most recent CPU on which task pid executed */
LIBINTERFACE int cpuset_latestcpu(pid_t pid)
{
    char buf[PATH_MAX];
    char *bp;
    int fd = -1;
    int cpu = -1;

    if (pid == 0)
	snprintf(buf, sizeof(buf), "/proc/self/stat");
    else
	snprintf(buf, sizeof(buf), "/proc/%d/stat", pid);

    if ((fd = open(buf, O_RDONLY)) < 0) {
	errno = ESRCH;		// no such file -> no such PID(process)
	goto err;
    }
    if (read(fd, buf, sizeof(buf)) < 1)
	goto err;
    close(fd);

    bp = strrchr(buf, ')');
    if (bp)
	sscanf(bp + 1, "%*s %*u %*u %*u %*u %*u %*u %*u " "%*u %*u %*u %*u %*u %*u %*u %*u %*u %*u " "%*u %*u %*u %*u %*u %*u %*u %*u %*u %*u " "%*u %*u %*u %*u %*u %*u %*u %*u %u",	/* 37th field past ')' */
	       &cpu);
    if (cpu < 0)
	errno = EINVAL;
    return cpu;
  err:
    if (fd >= 0)
	close(fd);
    return -1;
}
LIBINTERFACE int cpuset_getcpuset(const char *name, struct cpuset *cs)
{
	char buf[PATH_MAX];
	char dotchar[PATH_MAX];
	FILE *fp;
	fullpath2(buf, sizeof(buf), name, CPUSFILE );
  	if (!is_exist(buf))
   	 {
		dotchar[0] = '.';
		dotchar[1] = 0;
		strcat(dotchar, name);
		fullpath2(buf, sizeof(buf), dotchar,CPUSFILE); 
   	 }
	if ( (fp = fopen(buf, "r")) == NULL)
		return 0;
	fgets(cs->cpulist, PATH_MAX, fp);
	cs->cpulist[strlen(cs->cpulist) - 1] = '\0'; //\n -> \0
	fullpath2(buf, sizeof(buf), name, MEMSFILE );
	fgets(cs->memlist, PATH_MAX, fp);
	cs->memlist[strlen(cs->cpulist) - 1] = '\0'; //\n -> \0
	strcpy(cs->name, name);
	fclose(fp);
	return 1;
}

LIBINTERFACE int cpuset_getcpulist(pid_t pid, char *cpulist)
{
    char path[PATH_MAX];
    FILE *fp = NULL;
    char *buf = NULL;
    int fsize;
    /* check where am I */
    if ((buf = cpuset_getcpusetpath(pid, path, PATH_MAX)) == NULL)
	goto err;
    /* in ROOT cpuset */
    if (*buf == '/') {
	get_online_cpus();
	strcpy(cpulist, online_cpus_ptr);
	return 0;
    }
    /* not in ROOT cpuset */
    snprintf(path, PATH_MAX, "/proc/%d/status", pid);
    if ((fp = fopen(path, "r")) == NULL)
	goto err_on_open;
    fsize = filesize(fp);
    if ((buf = malloc(fsize)) == NULL)
	goto err;
    while (fgets(buf, fsize, fp)) {
	if (strprefix(buf, cpumask_list_prefix)) {
	    sscanf(buf, "%*s\t%s", cpulist);
	    return 0;
	}
    }
  err_on_open:
    errno = ESRCH;		// no such pid
  err:
    return -1;
}

static int audit_lbarea_mask(struct cpuset *cs)
{
    unsigned long *bitmaskp = malloc(cpumask_sz / sizeof(unsigned long));
    unsigned long *valid_cpus_maskp = malloc(cpumask_sz / sizeof(unsigned long));
//    unsigned long *hide_cpus_maskp  = malloc(cpumask_sz / sizeof(unsigned long));
    char *hide_cpulist = malloc(cpumask_sz);
    char *hide_cpulist_back;
    char path[PATH_MAX];
    /* compare cs->cpulist is legal or not */
    bitmap_parselist(cs->cpulist, bitmaskp, cpumask_sz);
    bitmap_parselist(online_cpus_ptr, valid_cpus_maskp, cpumask_sz);
    if (!bitmap_subset(bitmaskp, valid_cpus_maskp, cpumask_sz)) {
        errno = ENOCPU;
        goto err;
    }
    /* get ..hide/cpuset.cpus and make a backup */
    fullpath2(path, PATH_MAX, HIDE, CPUSFILE);
    read_string_file(path, hide_cpulist, PATH_MAX);
    hide_cpulist_back = strdup(hide_cpulist);
    bitmap_parselist(hide_cpulist, valid_cpus_maskp, cpumask_sz);
    if (!bitmap_subset(bitmaskp, valid_cpus_maskp, cpumask_sz)) {
        errno = ENOCPU;
        goto err;
    }
    /* get new mask for ..hide */
    bitmap_andnot(valid_cpus_maskp, valid_cpus_maskp, bitmaskp,
		  cpumask_sz);
    /* convert bitmask to string */
    bitmap_scnlistprintf(hide_cpulist, cpumask_sz, valid_cpus_maskp,
			 cpumask_sz);
    /* write to ..hide/cpuset.cpus */
    write_string_file(path, hide_cpulist);
    /* if exclude, set the flag: EXCLUDEFILE */
    if (cs->lbarea && cpuset_mark_lbarea(cs->name)<0)
    {   
        /* error happened, restore the oringial cpulist */
        write_string_file(path, hide_cpulist_back);
        errno = ENOCPU;
        goto err;
    }

    return 0;
  err:
    free(hide_cpulist);
    return -1;
}

static int audit_normal_mask(char *mask)
{
	return 0;
}

/*
   check cs valid or not
   return 0 on success
   return -1 if name or cpulist is invalid
AUDIT:
	1) not allowed to create cpuset starting with '.' without lbarea set;
	2) if lbarea set, check CPUSETMNT/cpuset.sched_load_balance is set or not;
	3) check .hide cpuset and remask it
*/
static int audit(struct cpuset *cs)
{
    char path[PATH_MAX];
    int ret;

    get_online_cpus();
    update_mask_sizes();

    if (cs->name[0] == '.' && !cs->lbarea)
		return -1;		// name start with '.' but not a lbarea cpsuet
    if (cs->lbarea) {
		fullpath(path, PATH_MAX, HIDE);
		/* if not exist ..hide, create one with normal cpuset */
		if (!is_exist(path)) {
			mkdir(path, 0755);	// ROOT/..hide
			fullpath(path, PATH_MAX, HIDE "/" CPUSFILE);
			write_string_file(path, online_cpus_ptr);
            /* tell bind to bind all tasks */
            initial = 1;
            cpuset_bind_cpuset_tasks("/", HIDE);
            /* errno changed, roll back to 0 */
            errno = 0;
		}
		ret = audit_lbarea_mask(cs);
    } else
		ret = audit_normal_mask(cs->cpulist);
    return ret;

}

/*
DESC:   create or modify a cpuset 
RET:	return 0 on success, -1 if error happened
	errorno changed
*/
static int create(const char *relpath)
{
    char buf[PATH_MAX];
    fullpath(buf, sizeof(buf), relpath);
    if (mkdir(buf, 0755) < 0)
    	return -1;
    return 0;
}

/*
 	create a cpuset 
*/
LIBINTERFACE int cpuset_create(struct cpuset *cs, void *nouse)
{
    char path[PATH_MAX];
    char path2[PATH_MAX];
    int ret;

    if (!cpumask_sz) {
	update_mask_sizes();
	/* get the global allowed mask */
	if (get_allowed_p() < 0) {
	    /* no cpuset mounted or else, oringial errno == ENOENT,
	       should change to ENODEV */
	    errno = ENODEV;
	    goto err;
        }
    }
	/* check EXIST name or .name */
	snprintf(path, PATH_MAX, ".%s", cs->name);	/* add . (path1 as temp buf */
    fullpath(path2, PATH_MAX, path);

    strcpy(path, cs->name);
    fullpath(path, PATH_MAX, cs->name);
    if (is_exist(path) || is_exist(path2))	//errno is set
    {
        errno = EEXIST;
        goto err;
    }
    /* create cpuset; if error errno already set */
    if (cs->lbarea)
    {
        /* add '.' ahead */
        snprintf(path, PATH_MAX, ".%s",cs->name);
        strncpy(cs->name, path, PATH_MAX);
        ret = create(cs->name);
    }else {
        ret = create(cs->name);
    }
    if (ret < 0) goto err;
    /* now created the new cpuset, put the mems and cpulist to it's file */
    if (write_mems(cs) < 0 || write_cpus(cs) < 0) {
        if (errno == EINVAL)
            errno = ENOCPU;	// no cpu available or cpu error 
    	goto invalid;
    }
    /* name or cpulist invalid */
    if (audit(cs) < 0)
        goto invalid;		
    return 0;
invalid:
    cpuset_del_byname(cs->name, 1);	//delete it
err:
    return -1;
}
int cpuset_exec(char **name, int cpu)
{
    char buf[PATH_MAX];
    unsigned long *valid_cpus_maskp = 	malloc(cpumask_sz / sizeof(unsigned long));
    get_online_cpus();
    update_mask_sizes();
//	printf("online:%s\n", online_cpus_ptr);
    bitmap_parselist(online_cpus_ptr, valid_cpus_maskp, cpumask_sz);
    if (cpu < 0 || cpu>cpumask_sz || !test_bit(cpu, valid_cpus_maskp)){
		errno = ENOCPU;
		goto err;
    }

/* everything goes ok, fork and exec */
if (fork() == 0 )	/* child execute the program */
{
	snprintf(buf, PATH_MAX, "%d", cpu);
	if (write_string_file(EXECON, buf) < 0)
	{
		if (errno == ENOENT)
				errno = ENOSYS;	/* proc/PID/execon not exist */
		else
				errno = EINVAL;
		goto err;	/* errno already set */
	}
	execvp(name[0], name);
	/* shouldn't come here */
	fprintf(stderr, "%s: %s\n", name[0], strerror(errno));
	exit(-1);
}
	return 0;
err:
	return -1;
}
